{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pathlib\n",
    "import yaml\n",
    "import re\n",
    "import os\n",
    "import sys\n",
    "\n",
    "from json import load, dump\n",
    "from typing import Set\n",
    "from os.path import join, exists, basename\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOKEN = input('github token')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./resources/supernova.yaml', 'r') as fp:\n",
    "    data = yaml.safe_load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_names = []\n",
    "repo_name_map = {}\n",
    "\n",
    "for item in data:\n",
    "    if item['githubURL'] != '':\n",
    "        parts = item['githubURL'].split('/')\n",
    "        cur_repo_name = '/'.join(parts[-2:])\n",
    "        repo_name_map[cur_repo_name] = item['name']\n",
    "        repo_names.append(cur_repo_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_star_response(search_string):\n",
    "    # Query the stars\n",
    "    URL = \"https://api.github.com/graphql\"\n",
    "    USER = \"xiaohk\"\n",
    "\n",
    "    query = \"\"\"\n",
    "    query {\n",
    "    rateLimit {\n",
    "        limit\n",
    "        cost\n",
    "        remaining\n",
    "    }\n",
    "    search(query: \"SEARCH_STRING\", type: REPOSITORY, first: 100) {\n",
    "        edges {\n",
    "        node {\n",
    "            ... on Repository {\n",
    "            owner {\n",
    "                login\n",
    "            }\n",
    "            name\n",
    "            stargazers {\n",
    "                totalCount\n",
    "            }\n",
    "            defaultBranchRef {\n",
    "                target {\n",
    "                    ... on Commit {\n",
    "                    history(first: 1) {\n",
    "                        edges {\n",
    "                        node {\n",
    "                            committedDate\n",
    "                        }\n",
    "                        }\n",
    "                        totalCount\n",
    "                        pageInfo {\n",
    "                            endCursor\n",
    "                        }\n",
    "                    }\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "            }\n",
    "        }\n",
    "        }\n",
    "    }\n",
    "    }\n",
    "    \"\"\"\n",
    "    query = query.replace(\"SEARCH_STRING\", search_string)\n",
    "\n",
    "    response = requests.post(\n",
    "        URL, json={\"query\": query}, auth=(USER, TOKEN), timeout=600\n",
    "    )\n",
    "\n",
    "    if not response.ok:\n",
    "        print(\"Request error!\")\n",
    "        print(response)\n",
    "        print(response.text)\n",
    "\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a query search string to query only related users\n",
    "i = 0\n",
    "gap = 100\n",
    "star_responses = []\n",
    "\n",
    "while i < len(repo_names):\n",
    "    search_string = \"\"\n",
    "    for name in repo_names[i: i + gap]:\n",
    "        search_string += f\"repo:{name} \"\n",
    "\n",
    "    response = get_star_response(search_string)\n",
    "    star_responses.append(response.json())\n",
    "    i += gap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "star_counts = {}\n",
    "cursor_info = {}\n",
    "repo_date_map = {}\n",
    "\n",
    "def parse_star_response(star_response):\n",
    "    # Parse the query result\n",
    "    data = star_response[\"data\"]\n",
    "\n",
    "\n",
    "    for node in data[\"search\"][\"edges\"]:\n",
    "        node = node[\"node\"]\n",
    "\n",
    "        # Get the query results\n",
    "        owner = node[\"owner\"][\"login\"]\n",
    "        name = node[\"name\"]\n",
    "        star = node[\"stargazers\"][\"totalCount\"]\n",
    "\n",
    "        repo_name = f\"{owner}/{name}\"\n",
    "        star_counts[repo_name] = star\n",
    "\n",
    "        # Get the cursor info\n",
    "        info = {\n",
    "            'committedDate': node[\"defaultBranchRef\"][\"target\"]['history'][\"edges\"][0][\"node\"][\"committedDate\"],\n",
    "            'totalCount': node[\"defaultBranchRef\"][\"target\"]['history'][\"totalCount\"],\n",
    "            'endCursor': node[\"defaultBranchRef\"][\"target\"]['history'][\"pageInfo\"][\"endCursor\"]\n",
    "        }\n",
    "        cursor_info[repo_name] = info\n",
    "\n",
    "for star_response in star_responses:\n",
    "    parse_star_response(star_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 135/135 [00:08<00:00, 15.11it/s]\n"
     ]
    }
   ],
   "source": [
    "# Get the first commit date\n",
    "query = \"\"\"\n",
    "query ($name: String!, $owner: String!){\n",
    "  repository(name: $name, owner: $owner) {\n",
    "    defaultBranchRef {\n",
    "      target {\n",
    "        ... on Commit {\n",
    "          history(first: 1, after: %s) {\n",
    "            nodes {\n",
    "              message\n",
    "              committedDate\n",
    "              authoredDate\n",
    "              oid\n",
    "              author {\n",
    "                email\n",
    "                name\n",
    "              }\n",
    "            }\n",
    "            totalCount\n",
    "            pageInfo {\n",
    "              endCursor\n",
    "            }\n",
    "          }\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "  }\n",
    "}\n",
    "\"\"\"\n",
    "\n",
    "def getDate(cursor):\n",
    "    r = requests.post(\"https://api.github.com/graphql\",\n",
    "        headers = {\n",
    "            \"Authorization\": f\"Bearer {TOKEN}\"\n",
    "        },\n",
    "        json = {\n",
    "            \"query\": query % cursor,\n",
    "            \"variables\": {\n",
    "                \"name\": name,\n",
    "                \"owner\": owner,\n",
    "            }\n",
    "        })\n",
    "    return r.json()[\"data\"][\"repository\"][\"defaultBranchRef\"][\"target\"][\"history\"][\"nodes\"][0][\"committedDate\"]\n",
    "\n",
    "for key in tqdm(cursor_info):\n",
    "  if key in repo_date_map:\n",
    "    continue\n",
    "\n",
    "  info = cursor_info[key]\n",
    "  owner, name = key.split('/')\n",
    "  totalCount = info['totalCount']\n",
    "\n",
    "  if totalCount > 1:\n",
    "    cursor = info['endCursor'].split(' ')\n",
    "    cursor[1] = str(totalCount - 2)\n",
    "    date = getDate(f\"\\\"{' '.join(cursor)}\\\"\")\n",
    "    repo_date_map[key] = date\n",
    "  else:\n",
    "    repo_date_map[key] = info['committedDate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remap the repo name back to their data source name\n",
    "\n",
    "star_info = {}\n",
    "for key in star_counts:\n",
    "    name = repo_name_map[key]\n",
    "    star_info[name] = {\n",
    "        'star': star_counts[key],\n",
    "        'date': repo_date_map[key]\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "star_info\n",
    "\n",
    "dump(star_info, open('./resources/star_info.json', 'w'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
